%load_ext autoreload
%autoreload 2
import os
MOMAPS_HOME = '/home/labs/hornsteinlab/Collaboration/MOmaps_Noam/MOmaps'
MOMAPS_DATA_HOME = '/home/labs/hornsteinlab/Collaboration/MOmaps'
LOGS_PATH = os.path.join(MOMAPS_DATA_HOME, "outputs/preprocessing/spd/logs/preprocessing_Dec2023")
PLOT_PATH = os.path.join(MOMAPS_HOME, 'src', 'preprocessing', 'notebooks','figures','Neurons')
os.chdir(MOMAPS_HOME)
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
plt.rcParams["image.cmap"] = "Set1"
from src.common.lib.preprocessing_utils import rescale_intensity
from src.common.lib.images_qc import *
import contextlib
import io
import matplotlib
import warnings
warnings.filterwarnings('ignore', category=pd.core.common.SettingWithCopyWarning)
from src.common.lib.qc_config_tmp import *
from src.common.lib.image_sampling_utils import *
from matplotlib.colors import LinearSegmentedColormap
from IPython.display import display, Javascript
df = log_files_qc(LOGS_PATH)
df = df[df.cell_line_cond!='SCNA']
df_dapi = df[df.marker=='DAPI']
df_target = df[df.marker!='DAPI']
reading logs of batch4_original_with_brenner reading logs of batch3_original_with_brenner reading logs of batch9_original_with_brenner reading logs of batch6_original_with_brenner reading logs of batch7_original_with_brenner reading logs of batch5_original_with_brenner reading logs of batch8_original_with_brenner Total of 12 files were read. Before dup handeling (398635, 22) After duplication removal #1: (389910, 23) After duplication removal #2: (384581, 23)
# choose batches
batches = [f'batch{i}' for i in range(3,10)]
batches
['batch3', 'batch4', 'batch5', 'batch6', 'batch7', 'batch8', 'batch9']
root_directory_raw = os.path.join(MOMAPS_DATA_HOME, 'input', 'images', 'raw', 'SpinningDisk')
batches_raw = [batch.replace("_16bit_no_downsample","") for batch in batches]
raws = run_validate_folder_structure(root_directory_raw, False, panels, markers,PLOT_PATH,marker_info,
cell_lines_to_cond, reps, cell_lines_for_disp, expected_dapi_raw,
batches=batches_raw)
batch3 Folder structure is invalid. Missing paths: /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/raw/SpinningDisk/batch3/FUSHomozygous/panelA /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/raw/SpinningDisk/batch3/FUSHomozygous/panelB /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/raw/SpinningDisk/batch3/FUSHomozygous/panelC /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/raw/SpinningDisk/batch3/TDP43/panelA /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/raw/SpinningDisk/batch3/TDP43/panelB /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/raw/SpinningDisk/batch3/TDP43/panelC /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/raw/SpinningDisk/batch3/TBK1/panelA /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/raw/SpinningDisk/batch3/TBK1/panelB /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/raw/SpinningDisk/batch3/TBK1/panelC /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/raw/SpinningDisk/batch3/WT/panelA /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/raw/SpinningDisk/batch3/WT/panelB /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/raw/SpinningDisk/batch3/WT/panelC /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/raw/SpinningDisk/batch3/FUSRevertant/panelA /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/raw/SpinningDisk/batch3/FUSRevertant/panelB /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/raw/SpinningDisk/batch3/FUSRevertant/panelC /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/raw/SpinningDisk/batch3/OPTN/panelA /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/raw/SpinningDisk/batch3/OPTN/panelB /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/raw/SpinningDisk/batch3/OPTN/panelC /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/raw/SpinningDisk/batch3/FUSHeterozygous/panelA /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/raw/SpinningDisk/batch3/FUSHeterozygous/panelB /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/raw/SpinningDisk/batch3/FUSHeterozygous/panelC Some files are bad: CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_output CellProfiler_output CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count CellProfiler_DAPI-count Total Sites: 38527
======== batch4 Folder structure is valid. No bad files are found. Total Sites: 57600
======== batch5 Folder structure is valid. No bad files are found. Total Sites: 57600
======== batch6 Folder structure is valid. No bad files are found. Total Sites: 57600
======== batch7 Folder structure is valid. No bad files are found. Total Sites: 57600
======== batch8 Folder structure is valid. No bad files are found. Total Sites: 57600
======== batch9 Folder structure is valid. No bad files are found. Total Sites: 57600
======== ====================
root_directory_proc = os.path.join(MOMAPS_DATA_HOME, 'input', 'images', 'processed', 'spd2',
'SpinningDisk')
procs = run_validate_folder_structure(root_directory_proc, True, panels, markers,PLOT_PATH,marker_info,
cell_lines_to_cond, reps, cell_lines_for_disp, expected_dapi_raw,
batches=batches)
batch3 Folder structure is invalid. Missing paths: /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/FUSHomozygous/Untreated/G3BP1 /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/FUSHomozygous/Untreated/NONO /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/FUSHomozygous/Untreated/SQSTM1 /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/FUSHomozygous/Untreated/KIF5A /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/FUSHomozygous/Untreated/TDP43 /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/FUSHomozygous/Untreated/FMRP /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/FUSHomozygous/Untreated/PURA /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/FUSHomozygous/Untreated/CD41 /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/FUSHomozygous/Untreated/Phalloidin /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/TDP43/Untreated/G3BP1 /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/TDP43/Untreated/NONO /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/TDP43/Untreated/SQSTM1 /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/TDP43/Untreated/KIF5A /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/TDP43/Untreated/TDP43 /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/TDP43/Untreated/FMRP /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/TDP43/Untreated/PURA /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/TDP43/Untreated/CD41 /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/TDP43/Untreated/Phalloidin /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/TBK1/Untreated/G3BP1 /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/TBK1/Untreated/NONO /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/TBK1/Untreated/SQSTM1 /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/TBK1/Untreated/KIF5A /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/TBK1/Untreated/TDP43 /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/TBK1/Untreated/FMRP /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/TBK1/Untreated/PURA /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/TBK1/Untreated/CD41 /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/TBK1/Untreated/Phalloidin /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/WT/Untreated/G3BP1 /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/WT/Untreated/NONO /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/WT/Untreated/SQSTM1 /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/WT/Untreated/KIF5A /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/WT/Untreated/TDP43 /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/WT/Untreated/FMRP /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/WT/Untreated/PURA /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/WT/Untreated/CD41 /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/WT/Untreated/Phalloidin /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/WT/stress/G3BP1 /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/WT/stress/NONO /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/WT/stress/SQSTM1 /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/WT/stress/KIF5A /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/WT/stress/TDP43 /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/WT/stress/FMRP /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/WT/stress/PURA /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/WT/stress/CD41 /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/WT/stress/Phalloidin /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/FUSRevertant/Untreated/G3BP1 /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/FUSRevertant/Untreated/NONO /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/FUSRevertant/Untreated/SQSTM1 /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/FUSRevertant/Untreated/KIF5A /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/FUSRevertant/Untreated/TDP43 /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/FUSRevertant/Untreated/FMRP /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/FUSRevertant/Untreated/PURA /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/FUSRevertant/Untreated/CD41 /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/FUSRevertant/Untreated/Phalloidin /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/OPTN/Untreated/G3BP1 /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/OPTN/Untreated/NONO /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/OPTN/Untreated/SQSTM1 /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/OPTN/Untreated/KIF5A /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/OPTN/Untreated/TDP43 /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/OPTN/Untreated/FMRP /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/OPTN/Untreated/PURA /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/OPTN/Untreated/CD41 /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/OPTN/Untreated/Phalloidin /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/FUSHeterozygous/Untreated/G3BP1 /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/FUSHeterozygous/Untreated/NONO /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/FUSHeterozygous/Untreated/SQSTM1 /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/FUSHeterozygous/Untreated/KIF5A /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/FUSHeterozygous/Untreated/TDP43 /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/FUSHeterozygous/Untreated/FMRP /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/FUSHeterozygous/Untreated/PURA /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/FUSHeterozygous/Untreated/CD41 /home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3/FUSHeterozygous/Untreated/Phalloidin No bad files are found. Total Sites: 31504
======== batch4 Folder structure is valid. No bad files are found. Total Sites: 44040
======== batch5 Folder structure is valid. No bad files are found. Total Sites: 47883
======== batch6 Folder structure is valid. No bad files are found. Total Sites: 55828
======== batch7 Folder structure is valid. No bad files are found. Total Sites: 56071
======== batch8 Folder structure is valid. No bad files are found. Total Sites: 55505
======== batch9 Folder structure is valid. No bad files are found. Total Sites: 55574
======== ====================
display_diff(batches, raws, procs, PLOT_PATH)
batch3
======== batch4
======== batch5
======== batch6
======== batch7
======== batch8
======== batch9
========
#for batch in list(range(3,9)) + ['7_16bit','8_16bit','9_16bit']:
for batch in batches:
with contextlib.redirect_stdout(io.StringIO()):
var = sample_and_calc_variance(root_directory_proc, batch,
sample_size_per_markers=200, num_markers=26)
print(f'{batch} var: ',var)
batch3 var: 0.015569925950291956 batch4 var: 0.014111859890173273 batch5 var: 0.014974081250157117 batch6 var: 0.016460123067249793 batch7 var: 0.016082496154033385 batch8 var: 0.016435503753641743 batch9 var: 0.016846129673614187
By order of filtering
Percentage out of the total sites
dapi_filter_by_brenner = show_site_survival_dapi_brenner(df_dapi,batches, line_colors, panels)
Percentage out of the sites that passed the previous filter. In parenthesis are absolute values.
A site will be filtered out if Cellpose found 0 cells in it.
dapi_filter_by_cellpose = show_site_survival_dapi_cellpose(df_dapi, batches, dapi_filter_by_brenner, line_colors, panels)
Percentage out of the sites that passed the previous filter. In parenthesis are absolute values.
A site will be filtered out if after tiling, no tile is containing at least 85% of a cell that Cellpose detected.
dapi_filter_by_tiling=show_site_survival_dapi_tiling(df_dapi, batches, dapi_filter_by_cellpose, line_colors, panels)
Percentage out of the sites that passed the previous filter. In parenthesis are absolute values (if different than the percentages).
show_site_survival_target_brenner(df_dapi, df_target, dapi_filter_by_tiling)
names = ['Total number of tiles', 'Total number of whole cells']
stats = ['n_valid_tiles','site_whole_cells_counts_sum','site_cell_count']
total_sum = calc_total_sums(df_target, df_dapi, stats)
for stat, name in zip(stats[:2], names):
to_heatmap = total_sum.rename(columns={stat:'index'})
plot_filtering_heatmap(to_heatmap, extra_index='marker', vmin=None, vmax=None,
xlabel = name, show_sum=True, figsize=(6,8))
show_total_sum_tables(total_sum)
| n_valid_tiles | % valid tiles | site_whole_cells_counts_sum | site_cell_count | |
|---|---|---|---|---|
| batch3 | ||||
| count | 249.000000 | 249.000000 | 249.000000 | 249.000000 |
| mean | 540.409639 | 5.404096 | 583.136546 | 1527.702811 |
| std | 302.460196 | 3.024602 | 331.053130 | 874.566713 |
| min | 1.000000 | 0.010000 | 1.000000 | 4.000000 |
| 25% | 357.000000 | 3.570000 | 381.000000 | 954.000000 |
| 50% | 534.000000 | 5.340000 | 570.000000 | 1557.000000 |
| 75% | 785.000000 | 7.850000 | 849.000000 | 2173.000000 |
| max | 1176.000000 | 11.760000 | 1306.000000 | 3471.000000 |
| sum | 134562.000000 | NaN | 145201.000000 | 380398.000000 |
| expected_count | 450.000000 | 450.000000 | 450.000000 | 450.000000 |
| n_valid_tiles | % valid tiles | site_whole_cells_counts_sum | site_cell_count | |
|---|---|---|---|---|
| batch4 | ||||
| count | 393.000000 | 393.000000 | 393.000000 | 393.000000 |
| mean | 514.633588 | 5.146336 | 558.643766 | 1292.000000 |
| std | 334.876650 | 3.348766 | 370.766182 | 874.038343 |
| min | 6.000000 | 0.060000 | 6.000000 | 9.000000 |
| 25% | 196.000000 | 1.960000 | 203.000000 | 469.000000 |
| 50% | 474.000000 | 4.740000 | 511.000000 | 1265.000000 |
| 75% | 855.000000 | 8.550000 | 934.000000 | 2070.000000 |
| max | 1158.000000 | 11.580000 | 1294.000000 | 3027.000000 |
| sum | 202251.000000 | NaN | 219547.000000 | 507756.000000 |
| expected_count | 450.000000 | 450.000000 | 450.000000 | 450.000000 |
| n_valid_tiles | % valid tiles | site_whole_cells_counts_sum | site_cell_count | |
|---|---|---|---|---|
| batch5 | ||||
| count | 399.000000 | 399.000000 | 399.000000 | 399.000000 |
| mean | 572.561404 | 5.725614 | 624.045113 | 1452.187970 |
| std | 330.515747 | 3.305157 | 365.806156 | 873.976617 |
| min | 1.000000 | 0.010000 | 1.000000 | 4.000000 |
| 25% | 251.000000 | 2.510000 | 270.000000 | 618.000000 |
| 50% | 590.000000 | 5.900000 | 626.000000 | 1522.000000 |
| 75% | 857.500000 | 8.575000 | 939.000000 | 2214.500000 |
| max | 1157.000000 | 11.570000 | 1281.000000 | 2988.000000 |
| sum | 228452.000000 | NaN | 248994.000000 | 579423.000000 |
| expected_count | 450.000000 | 450.000000 | 450.000000 | 450.000000 |
| n_valid_tiles | % valid tiles | site_whole_cells_counts_sum | site_cell_count | |
|---|---|---|---|---|
| batch6 | ||||
| count | 400.00000 | 400.000000 | 400.000000 | 400.000000 |
| mean | 710.73250 | 7.107325 | 757.992500 | 2006.835000 |
| std | 176.86102 | 1.768610 | 192.619517 | 517.149942 |
| min | 73.00000 | 0.730000 | 74.000000 | 214.000000 |
| 25% | 589.00000 | 5.890000 | 625.000000 | 1645.750000 |
| 50% | 717.50000 | 7.175000 | 765.000000 | 2056.500000 |
| 75% | 834.00000 | 8.340000 | 874.000000 | 2363.000000 |
| max | 1095.00000 | 10.950000 | 1189.000000 | 2996.000000 |
| sum | 284293.00000 | NaN | 303197.000000 | 802734.000000 |
| expected_count | 450.00000 | 450.000000 | 450.000000 | 450.000000 |
| n_valid_tiles | % valid tiles | site_whole_cells_counts_sum | site_cell_count | |
|---|---|---|---|---|
| batch7 | ||||
| count | 400.000000 | 400.000000 | 400.0000 | 400.000000 |
| mean | 697.820000 | 6.978200 | 743.9700 | 1822.452500 |
| std | 172.560149 | 1.725601 | 188.6011 | 466.101587 |
| min | 365.000000 | 3.650000 | 390.0000 | 867.000000 |
| 25% | 572.000000 | 5.720000 | 612.2500 | 1483.750000 |
| 50% | 698.000000 | 6.980000 | 737.5000 | 1808.500000 |
| 75% | 827.500000 | 8.275000 | 881.2500 | 2155.000000 |
| max | 1178.000000 | 11.780000 | 1261.0000 | 3164.000000 |
| sum | 279128.000000 | NaN | 297588.0000 | 728981.000000 |
| expected_count | 450.000000 | 450.000000 | 450.0000 | 450.000000 |
| n_valid_tiles | % valid tiles | site_whole_cells_counts_sum | site_cell_count | |
|---|---|---|---|---|
| batch8 | ||||
| count | 400.000000 | 400.000000 | 400.000000 | 400.000000 |
| mean | 646.270000 | 6.462700 | 685.905000 | 1736.702500 |
| std | 174.682834 | 1.746828 | 188.968221 | 498.600366 |
| min | 73.000000 | 0.730000 | 79.000000 | 176.000000 |
| 25% | 532.500000 | 5.325000 | 559.000000 | 1419.500000 |
| 50% | 656.000000 | 6.560000 | 694.500000 | 1753.000000 |
| 75% | 763.000000 | 7.630000 | 812.000000 | 2093.250000 |
| max | 1086.000000 | 10.860000 | 1189.000000 | 2996.000000 |
| sum | 258508.000000 | NaN | 274362.000000 | 694681.000000 |
| expected_count | 450.000000 | 450.000000 | 450.000000 | 450.000000 |
| n_valid_tiles | % valid tiles | site_whole_cells_counts_sum | site_cell_count | |
|---|---|---|---|---|
| batch9 | ||||
| count | 398.000000 | 398.000000 | 398.000000 | 398.000000 |
| mean | 725.766332 | 7.257663 | 775.123116 | 2009.376884 |
| std | 197.438198 | 1.974382 | 215.173850 | 545.844936 |
| min | 92.000000 | 0.920000 | 97.000000 | 224.000000 |
| 25% | 577.000000 | 5.770000 | 611.250000 | 1641.250000 |
| 50% | 742.500000 | 7.425000 | 790.000000 | 2086.500000 |
| 75% | 879.500000 | 8.795000 | 939.500000 | 2411.500000 |
| max | 1205.000000 | 12.050000 | 1323.000000 | 3215.000000 |
| sum | 288855.000000 | NaN | 308499.000000 | 799732.000000 |
| expected_count | 450.000000 | 450.000000 | 450.000000 | 450.000000 |
| n valid tiles | % valid tiles | site_whole_cells_counts_sum | site_cell_count | |
|---|---|---|---|---|
| All batches | ||||
| count | 2.639000e+03 | 2639.000000 | 2.639000e+03 | 2.639000e+03 |
| mean | 6.351076e+02 | 6.351076 | 6.810868e+02 | 1.702806e+03 |
| std | 2.603559e+02 | 2.603559 | 2.841570e+02 | 7.242192e+02 |
| min | 1.000000e+00 | 0.010000 | 1.000000e+00 | 4.000000e+00 |
| 25% | 4.860000e+02 | 4.860000 | 5.140000e+02 | 1.289000e+03 |
| 50% | 6.640000e+02 | 6.640000 | 7.080000e+02 | 1.775000e+03 |
| 75% | 8.220000e+02 | 8.220000 | 8.815000e+02 | 2.231000e+03 |
| max | 1.205000e+03 | 12.050000 | 1.323000e+03 | 3.471000e+03 |
| sum | 1.676049e+06 | NaN | 1.797388e+06 | 4.493705e+06 |
| expected_count | 4.500000e+02 | 450.000000 | 4.500000e+02 | 4.500000e+02 |
df_no_empty_sites = df_dapi[df_dapi.n_valid_tiles !=0]
plot_cell_count(df_no_empty_sites, lines_order, custom_palette, y='site_cell_count_sum',
title='Cell Count Average per Site (from tiles)')
plot_cell_count(df_no_empty_sites, lines_order, custom_palette, y='site_whole_cells_counts_sum',
title='Whole Cell Count Average per Site')
plot_cell_count(df_no_empty_sites, lines_order, custom_palette, y='site_cell_count',
title='Cellpose Cell Count Average per Site')
plot_catplot(df_dapi, custom_palette,reps, x='n_valid_tiles', x_title='valid tiles count', batch_min=3, batch_max=9)
plot_hm(df_dapi, split_by='rep', rows='cell_line', columns='panel')
# For supp figure 1
plot_hm_combine_batches(df_dapi, batches=['batch7','batch8'],
reps = ['rep1','rep2'],rows='cell_line', columns='panel')
for batch in batches:
print(batch)
#batch_num = batch.replace('batch',"")
run_calc_hist_new(batch,cell_lines_for_disp, markers,
hist_sample=10,sample_size_per_markers=200, ncols=7, nrows=4)
print("="*30)
batch3
/home/labs/hornsteinlab/Collaboration/MOmaps_Noam/MOmaps/src/common/lib/images_qc.py:914: UserWarning: This figure includes Axes that are not compatible with tight_layout, so results might be incorrect. plt.tight_layout()
/home/labs/hornsteinlab/Collaboration/MOmaps/anaconda3/momaps_37/lib/python3.7/site-packages/IPython/core/pylabtools.py:151: UserWarning: This figure includes Axes that are not compatible with tight_layout, so results might be incorrect. fig.canvas.print_figure(bytes_io, **kw)
============================== batch4
============================== batch5
============================== batch6
============================== batch7
============================== batch8
============================== batch9
# save notebook as HTML ( the HTML will be saved in the same folder the original script is)
display(Javascript('IPython.notebook.save_checkpoint();'))
os.system('jupyter nbconvert --to html src/preprocessing/notebooks/no_ds/qc_report_newPP.ipynb')
# save notebook as HTML ( the HTML will be saved in the same folder the original script is)
display(Javascript('IPython.notebook.save_checkpoint();'))
os.system('jupyter nbconvert --to pdf src/preprocessing/notebooks/no_ds/qc_report_newPP.ipynb')
import pandoc